df.info()

print('Shape:', df.shape)
df.head()

df.describe()

check_missing_values(df)

# Create a skewness summary DataFrame
skew_summary = pd.DataFrame({
    'Feature': num_cols,
    'Skewness': [df[col].skew().round(2) for col in num_cols]
}).sort_values(by='Skewness', ascending=False)

skew_summary

for col in num_cols:
    histogram_boxplot(df, col, kde=True)

# Calculate default proportions
default_counts = df['BAD'].value_counts()
labels = ['No Default (0)', 'Default (1)']

# Plot pie chart
default_counts.plot(
    kind='pie',
    labels=labels,
    explode=[0.05, 0],
    autopct='%1.1f%%',
    startangle=90,
    colors=['#66b3ff', '#ff9999'],
    wedgeprops={'edgecolor': 'black'}
)

plt.title("Loan Repayment Status (BAD Variable)", fontsize=14)
plt.ylabel('')  
plt.axis('equal')  
plt.show()

# Value counts for REASON
reason_counts = df['REASON'].value_counts()
labels = reason_counts.index.tolist()
colors = ['#66b3ff', '#ff9999']  # Customize based on # of categories

# Pie chart
reason_counts.plot(
    kind='pie',
    labels=labels,
    autopct='%1.1f%%',
    explode=[0.05, 0],
    startangle=90,
    colors=colors,
    wedgeprops={'edgecolor': 'black'}
)

plt.title("Loan Reason Distribution", fontsize=14)
plt.ylabel('')
plt.axis('equal')  # Equal aspect ratio for perfect circle
plt.show()

# JOB -> Manager, Office, Sales, Self-employed, Professional/Executive, Other
print("JOB value counts:")
print(df['JOB'].value_counts())

sns.countplot(x='JOB', data=df)
plt.title("Job Type Distribution")
plt.xticks(rotation=45)
plt.show()

# Plot 1: JOB vs BAD with % annotations
plt.figure(figsize=(8, 4))
ax = sns.countplot(x='JOB', hue='BAD', data=df)

# Title and labels
plt.title("Default Rate by Job Type")
plt.xlabel("Job Category")
plt.ylabel("Count")
plt.xticks(rotation=45)
plt.legend(title="BAD", labels=["No Default (0)", "Default (1)"])

# Add % labels only for non-zero bars
for p in ax.patches:
    count = p.get_height()
    if count > 0:
        x = p.get_x() + p.get_width() / 2.0
        label = f"{100 * count / len(df):.1f}%"
        ax.annotate(label, (x, count), ha='center', va='bottom', fontsize=9)

plt.show()

# Plot 2: REASON vs BAD with % annotations
plt.figure(figsize=(8, 4))
ax = sns.countplot(x='REASON', hue='BAD', data=df)

# Title and labels
plt.title("Default Rate by Loan Reason")
plt.xlabel("Loan Reason")
plt.ylabel("Count")
plt.xticks(rotation=45)
plt.legend(title="BAD", labels=["No Default (0)", "Default (1)"])

# Add % labels only for non-zero bars
for p in ax.patches:
    count = p.get_height()
    if count > 0:
        x = p.get_x() + p.get_width() / 2.0
        label = f"{100 * count / len(df):.1f}%"
        ax.annotate(label, (x, count), ha='center', va='bottom', fontsize=9)

plt.show()

for col in num_cols:
    plt.figure(figsize=(7,4))
    sns.boxplot(data=df, x='BAD', y=col, hue='BAD', palette='Set1', dodge=False, legend=False)
    plt.title(f"{col} by Loan Default (BAD)")
    plt.xlabel("BAD (0 = No Default, 1 = Default)")
    plt.ylabel(col)
    plt.show()

# Build correlation matrix for numerical columns
corr = df[num_cols].corr()

# Plot the heatmap
plt.figure(figsize=(16,12))
sns.heatmap(corr, cmap='coolwarm', vmax=1, vmin=-1,
            annot=True, fmt=".2f",
            xticklabels=corr.columns,
            yticklabels=corr.columns)
plt.title('Correlation Matrix for Numerical Columns')
plt.show()

sns.pairplot(df, hue='BAD', diag_kind='kde', palette='Set1')
plt.suptitle('Pairplot of Features Colored by Default Status', y=1.02)
plt.show()

# Fill CATEGORICAL features with 'Other'
df['REASON'] = df['REASON'].fillna('Other')
df['JOB'] = df['JOB'].fillna('Other')

# Fill CREDIT HISTORY fields with 0 (assuming no issues)
for col in ['DEROG', 'DELINQ', 'NINQ']:
    df[col] = df[col].fillna(0)

# Fill skewed NUMERIC features with median to preserve central tendency and avoid outlier distortion
for col in ['MORTDUE', 'VALUE', 'CLAGE', 'CLNO', 'DEBTINC']:
    df[col] = df[col].fillna(df[col].median())

# Group-wise median imputation for YOJ based on JOB
df['YOJ'] = df.groupby('JOB')['YOJ'].transform(lambda x: x.fillna(x.median()))

print(df.isnull().sum()[df.isnull().sum() > 0])

# Check for outliers again
for col in cap_cols:
    plt.figure(figsize=(7,4))
    sns.boxplot(data=df, y=col)
    plt.title(f"Boxplot of {col} after IQR Capping")
    plt.show()

df.isnull().sum()

print("LOAN mean:", df['LOAN'].mean())
print("LOAN std:", df['LOAN'].std())

df.info()

# Split the data into features and target variable
X = df.drop(columns='BAD')
y = df['BAD']

# Ensure enough class diversity
print("Class distribution:\n", y.value_counts())

# Split the dataset into training and testing sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=44, stratify=y)

# Check the shape of the splits
print(f"X_train shape: {X_train.shape}")
print(f"X_test shape: {X_test.shape}")
print(f"y_train shape: {y_train.shape}")
print(f"y_test shape: {y_test.shape}")

print("Train class distribution:\n", y_train.value_counts())
print("Test class distribution:\n", y_test.value_counts())

# Evaluation on training set
print("Classification Report (Train):")
print(classification_report(y_train, y_train_pred))

# Round and print metrics
print("Accuracy:", round(accuracy_score(y_train, y_train_pred), 2))
print("Precision:", round(precision_score(y_train, y_train_pred), 2))
print("Recall:", round(recall_score(y_train, y_train_pred), 2))
print("F1 Score:", round(f1_score(y_train, y_train_pred), 2))
print("AUC Score:", round(roc_auc_score(y_train, y_train_proba), 2))

# Confusion Matrix with Labels
cm_train = confusion_matrix(y_train, y_train_pred)
sns.heatmap(cm_train, annot=True, fmt='d', cmap='Greens',
            xticklabels=['No Default', 'Default'],
            yticklabels=['No Default', 'Default'])
plt.title("Confusion Matrix - Logistic Regression (Train)")
plt.xlabel("Predicted Label")
plt.ylabel("True Label")
plt.show()

# ROC Curve
fpr_train, tpr_train, _ = roc_curve(y_train, y_train_proba)
plt.figure(figsize=(8, 6))
plt.plot(fpr_train, tpr_train, label=f"AUC = {roc_auc_score(y_train, y_train_proba):.2f}")
plt.plot([0, 1], [0, 1], linestyle='--', label='Random Guess')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC Curve - Logistic Regression (Train)')
plt.legend(loc='lower right')
plt.grid(True)
plt.show()

# Evaluation on test set
print("Classification Report (Test):")
print(classification_report(y_test, y_pred))

# Round and print individual metrics
print("Accuracy:", round(accuracy_score(y_test, y_pred), 2))
print("Precision:", round(precision_score(y_test, y_pred), 2))
print("Recall:", round(recall_score(y_test, y_pred), 2))
print("F1 Score:", round(f1_score(y_test, y_pred), 2))
print("AUC Score:", round(roc_auc_score(y_test, y_proba), 2))

# Confusion Matrix with Labels
cm = confusion_matrix(y_test, y_pred)
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', xticklabels=['No Default', 'Default'], yticklabels=['No Default', 'Default'])
plt.title("Confusion Matrix - Logistic Regression (Test)")
plt.xlabel("Predicted Label")
plt.ylabel("True Label")
plt.show()

# ROC Curve
fpr, tpr, _ = roc_curve(y_test, y_proba)
plt.figure(figsize=(8, 6))
plt.plot(fpr, tpr, label=f"AUC = {roc_auc_score(y_test, y_proba):.2f}")
plt.plot([0, 1], [0, 1], linestyle='--', label='Random Guessing')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC Curve - Logistic Regression (Test)')
plt.legend(loc='lower right')
plt.grid(True)
plt.show()

# Save metrics for comparison table later
logreg_metrics = {
    'model': 'Logistic Regression',
    'accuracy': round(accuracy_score(y_test, y_pred), 2),
    'precision': round(precision_score(y_test, y_pred), 2),
    'recall': round(recall_score(y_test, y_pred), 2),
    'f1': round(f1_score(y_test, y_pred), 2),
    'auc': round(roc_auc_score(y_test, y_proba), 2)
}

# Train set evaluation
print("Decision Tree Classification Report (Train):")
print(classification_report(y_train, y_train_pred_dt))

# Round and print metrics
print("Accuracy:", round(accuracy_score(y_train, y_train_pred_dt), 2))
print("Precision:", round(precision_score(y_train, y_train_pred_dt), 2))
print("Recall:", round(recall_score(y_train, y_train_pred_dt), 2))
print("F1 Score:", round(f1_score(y_train, y_train_pred_dt), 2))
print("AUC Score:", round(roc_auc_score(y_train, y_train_proba_dt), 2))

# Confusion Matrix with Labels
cm_train_dt = confusion_matrix(y_train, y_train_pred_dt)
sns.heatmap(cm_train_dt, annot=True, fmt='d', cmap='Greens',
            xticklabels=['No Default', 'Default'],
            yticklabels=['No Default', 'Default'])
plt.title("Confusion Matrix - Decision Tree (Train)")
plt.xlabel("Predicted Label")
plt.ylabel("True Label")
plt.show()

# ROC Curve
fpr_train_dt, tpr_train_dt, _ = roc_curve(y_train, y_train_proba_dt)
plt.figure(figsize=(8, 6))
plt.plot(fpr_train_dt, tpr_train_dt, label=f"AUC = {roc_auc_score(y_train, y_train_proba_dt):.2f}")
plt.plot([0, 1], [0, 1], linestyle='--', label='Random Guess')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC Curve - Decision Tree (Train)')
plt.legend(loc='lower right')
plt.grid(True)
plt.show()

# Test set evaluation
print("Classification Report (Test - Decision Tree):")
print(classification_report(y_test, y_pred_dt))

print("Accuracy:", round(accuracy_score(y_test, y_pred_dt), 2))
print("Precision:", round(precision_score(y_test, y_pred_dt), 2))
print("Recall:", round(recall_score(y_test, y_pred_dt), 2))
print("F1 Score:", round(f1_score(y_test, y_pred_dt), 2))
print("AUC Score:", round(roc_auc_score(y_test, y_proba_dt), 2))

# Confusion Matrix
cm_dt = confusion_matrix(y_test, y_pred_dt)
sns.heatmap(cm_dt, annot=True, fmt='d', cmap='Blues',
            xticklabels=['No Default', 'Default'],
            yticklabels=['No Default', 'Default'])
plt.title("Confusion Matrix - Decision Tree (Test)")
plt.xlabel("Predicted Label")
plt.ylabel("True Label")
plt.show()

# ROC Curve
fpr_dt, tpr_dt, _ = roc_curve(y_test, y_proba_dt)
plt.figure(figsize=(8, 6))
plt.plot(fpr_dt, tpr_dt, label=f"AUC = {roc_auc_score(y_test, y_proba_dt):.2f}")
plt.plot([0, 1], [0, 1], linestyle='--', label='Random Guessing')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC Curve - Decision Tree (Test)')
plt.legend(loc='lower right')
plt.grid(True)
plt.show()

# Save metrics
dt_metrics = {
    'model': 'Decision Tree',
    'accuracy': round(accuracy_score(y_test, y_pred_dt), 2),
    'precision': round(precision_score(y_test, y_pred_dt), 2),
    'recall': round(recall_score(y_test, y_pred_dt), 2),
    'f1': round(f1_score(y_test, y_pred_dt), 2),
    'auc': round(roc_auc_score(y_test, y_proba_dt), 2)
}

# Parameter grid
param_grid = {
    'classifier__max_depth': [3, 5, 7, 10, None],
    'classifier__min_samples_leaf': [1, 5, 10, 20],
    'classifier__min_samples_split': [2, 5, 10],
    'classifier__criterion': ['gini', 'entropy']
}

# Grid Search with Recall as scoring
grid_search = GridSearchCV(dt_pipeline, param_grid, cv=5, scoring='recall', n_jobs=-1)
grid_search.fit(X_train, y_train)

# Best Model
best_dt = grid_search.best_estimator_
print("Best Parameters:", grid_search.best_params_)

# Predict on training set
y_train_pred_dt = best_dt.predict(X_train)
y_train_proba_dt = best_dt.predict_proba(X_train)[:, 1]
# Evaluation on training set
print("Classification Report (Train - Tuned Decision Tree):")
print(classification_report(y_train, y_train_pred_dt))
# Round and print metrics
print("Accuracy:", round(accuracy_score(y_train, y_train_pred_dt), 2))
print("Precision:", round(precision_score(y_train, y_train_pred_dt), 2))
print("Recall:", round(recall_score(y_train, y_train_pred_dt), 2))
print("F1 Score:", round(f1_score(y_train, y_train_pred_dt), 2))
print("AUC Score:", round(roc_auc_score(y_train, y_train_proba_dt), 2))
# Confusion Matrix with Labels
cm_train = confusion_matrix(y_train, y_train_pred_dt)
sns.heatmap(cm_train, annot=True, fmt='d', cmap='Greens',
            xticklabels=['No Default', 'Default'],
            yticklabels=['No Default', 'Default'])
plt.title("Confusion Matrix - Decision Tree Tuned - Train")
plt.xlabel("Predicted Label")
plt.ylabel("True Label")
plt.show()
# ROC Curve
fpr_train, tpr_train, _ = roc_curve(y_train, y_train_proba_dt)
plt.figure(figsize=(8, 6))
plt.plot(fpr_train, tpr_train, label=f"AUC = {roc_auc_score(y_train, y_train_proba_dt):.2f}")
plt.plot([0, 1], [0, 1], linestyle='--', label='Random Guess')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC Curve - Decision Tree Tuned - Train')
plt.legend(loc='lower right')
plt.grid(True)
plt.show()

# Predict on test set
y_pred_dt = best_dt.predict(X_test)
y_proba_dt = best_dt.predict_proba(X_test)[:, 1]

# Classification report
print("Classification Report (Test - Tuned Decision Tree):")
print(classification_report(y_test, y_pred_dt))

print("Accuracy:", round(accuracy_score(y_test, y_pred_dt), 2))
print("Precision:", round(precision_score(y_test, y_pred_dt), 2))
print("Recall:", round(recall_score(y_test, y_pred_dt), 2))
print("F1 Score:", round(f1_score(y_test, y_pred_dt), 2))
print("AUC Score:", round(roc_auc_score(y_test, y_proba_dt), 2))

# Confusion Matrix
sns.heatmap(confusion_matrix(y_test, y_pred_dt), annot=True, fmt='d', cmap='Blues',
            xticklabels=['No Default', 'Default'],
            yticklabels=['No Default', 'Default'])
plt.title("Confusion Matrix - Decision Tree Tuned - Test")
plt.xlabel("Predicted Label")
plt.ylabel("True Label")
plt.show()

# ROC Curve
fpr, tpr, _ = roc_curve(y_test, y_proba_dt)
plt.figure(figsize=(8, 6))
plt.plot(fpr, tpr, label=f"AUC = {roc_auc_score(y_test, y_proba_dt):.2f}")
plt.plot([0, 1], [0, 1], linestyle='--', label='Random Guessing')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC Curve - Decision Tree Tuned - Test')
plt.legend()
plt.grid(True)
plt.show()
# Save metrics
dt_tuned_metrics = {
    'model': 'Tuned Decision Tree',
    'accuracy': round(accuracy_score(y_test, y_pred_dt), 2),
    'precision': round(precision_score(y_test, y_pred_dt), 2),
    'recall': round(recall_score(y_test, y_pred_dt), 2),
    'f1': round(f1_score(y_test, y_pred_dt), 2),
    'auc': round(roc_auc_score(y_test, y_proba_dt), 2)
}

# feature importance
importances = best_dt.named_steps['classifier'].feature_importances_
feature_names = num_cols + cat_cols
importances_df = pd.DataFrame({'Feature': feature_names, 'Importance': importances})
importances_df = importances_df.sort_values(by='Importance', ascending=False)
plt.figure(figsize=(10, 6))
sns.barplot(x='Importance', y='Feature', data=importances_df)
plt.title("Feature Importance - Tuned Decision Tree")
plt.xlabel("Importance")
plt.ylabel("Feature")
plt.show()

# Train set evaluation
print("Classification Report (Train - Random Forest):")
print(classification_report(y_train, y_train_pred_rf))

# Round and print metrics
print("Accuracy:", round(accuracy_score(y_train, y_train_pred_rf), 2))
print("Precision:", round(precision_score(y_train, y_train_pred_rf), 2))
print("Recall:", round(recall_score(y_train, y_train_pred_rf), 2))
print("F1 Score:", round(f1_score(y_train, y_train_pred_rf), 2))
print("AUC Score:", round(roc_auc_score(y_train, y_train_proba_rf), 2))

# Confusion Matrix with Labels
cm_train_rf = confusion_matrix(y_train, y_train_pred_rf)
sns.heatmap(cm_train_rf, annot=True, fmt='d', cmap='Greens',
            xticklabels=['No Default', 'Default'],
            yticklabels=['No Default', 'Default'])
plt.title("Confusion Matrix - Random Forest (Train)")
plt.xlabel("Predicted Label")
plt.ylabel("True Label")
plt.show()

# ROC Curve - Train
fpr_train_rf, tpr_train_rf, _ = roc_curve(y_train, y_train_proba_rf)
plt.figure(figsize=(8, 6))
plt.plot(fpr_train_rf, tpr_train_rf, label=f"AUC = {roc_auc_score(y_train, y_train_proba_rf):.2f}")
plt.plot([0, 1], [0, 1], linestyle='--', label='Random Guess')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC Curve - Random Forest (Train)')
plt.legend(loc='lower right')
plt.grid(True)
plt.show()

# Test Set Evaluation
print("Classification Report (Test - Random Forest):")
print(classification_report(y_test, y_pred_rf))

# Round and print metrics
print("Accuracy:", round(accuracy_score(y_test, y_pred_rf), 2))
print("Precision:", round(precision_score(y_test, y_pred_rf), 2))
print("Recall:", round(recall_score(y_test, y_pred_rf), 2))
print("F1 Score:", round(f1_score(y_test, y_pred_rf), 2))
print("AUC Score:", round(roc_auc_score(y_test, y_proba_rf), 2))

# Confusion Matrix - Test
sns.heatmap(confusion_matrix(y_test, y_pred_rf), annot=True, fmt='d', cmap='Blues',
            xticklabels=['No Default', 'Default'], yticklabels=['No Default', 'Default'])
plt.title("Confusion Matrix - Random Forest (Test)")
plt.xlabel("Predicted Label")
plt.ylabel("True Label")
plt.show()

# ROC Curve - Test
fpr_rf, tpr_rf, _ = roc_curve(y_test, y_proba_rf)
plt.figure(figsize=(8, 6))
plt.plot(fpr_rf, tpr_rf, label=f'AUC = {roc_auc_score(y_test, y_proba_rf):.2f}')
plt.plot([0, 1], [0, 1], linestyle='--', label='Random Guess')
plt.xlabel("False Positive Rate")
plt.ylabel("True Positive Rate")
plt.title("ROC Curve - Random Forest (Test)")
plt.legend()
plt.grid(True)
plt.show()

# Save metrics for comparison table later
rf_metrics = {
    'model': 'Random Forest',
    'accuracy': round(accuracy_score(y_test, y_pred_rf), 2),
    'precision': round(precision_score(y_test, y_pred_rf), 2),
    'recall': round(recall_score(y_test, y_pred_rf), 2),
    'f1': round(f1_score(y_test, y_pred_rf), 2),
    'auc': round(roc_auc_score(y_test, y_proba_rf), 2)
}

# Preprocessing (reusing structure from earlier)
rf_preprocessor = ColumnTransformer(transformers=[
    ('cat', OrdinalEncoder(), cat_cols)
], remainder='passthrough')

# Base classifier (can omit n_estimators here to grid it)
rf = RandomForestClassifier(random_state=42)

# Pipeline
rf_pipeline = Pipeline(steps=[
    ('preprocessor', rf_preprocessor),
    ('classifier', rf)
])

# Parameter grid based on best practices
rf_param_grid = {
    'classifier__n_estimators': [50, 100, 150, 200, 250, 300],
    'classifier__max_depth': [5, 10, 15, None],
    'classifier__min_samples_split': [2, 5, 10],
    'classifier__min_samples_leaf': [1, 2, 4],
    'classifier__max_features': ['sqrt', 'log2', None],
    'classifier__class_weight': [None, 'balanced']
}

# Grid search with recall scoring (can swap to f1 or auc later)
grid_search_rf = GridSearchCV(
    estimator=rf_pipeline,
    param_grid=rf_param_grid,
    scoring='recall',   # or 'f1' or 'roc_auc'
    cv=5,
    n_jobs=-1,
    verbose=1
)

# Run it
grid_search_rf.fit(X_train, y_train)

# Best model
best_rf = grid_search_rf.best_estimator_
print("Best Parameters:", grid_search_rf.best_params_)

# Evaluate on training set 
y_train_pred_rf = best_rf.predict(X_train)
y_train_proba_rf = best_rf.predict_proba(X_train)[:, 1]

# Evaluation on training set
print("Classification Report (Train - Tuned Random Forest):")
print(classification_report(y_train, y_train_pred_rf))
# Round and print metrics
print("Accuracy:", round(accuracy_score(y_train, y_train_pred_rf), 2))
print("Precision:", round(precision_score(y_train, y_train_pred_rf), 2))
print("Recall:", round(recall_score(y_train, y_train_pred_rf), 2))
print("F1 Score:", round(f1_score(y_train, y_train_pred_rf), 2))
print("AUC Score:", round(roc_auc_score(y_train, y_train_proba_rf), 2))

# Confusion Matrix with Labels
cm_train_rf = confusion_matrix(y_train, y_train_pred_rf)
sns.heatmap(cm_train_rf, annot=True, fmt='d', cmap='Greens',
            xticklabels=['No Default', 'Default'],
            yticklabels=['No Default', 'Default'])
plt.title("Confusion Matrix - Tuned Random Forest (Train)")
plt.xlabel("Predicted Label")
plt.ylabel("True Label")
plt.show()

# ROC Curve - Train
fpr_train_rf, tpr_train_rf, _ = roc_curve(y_train, y_train_proba_rf)
plt.figure(figsize=(8, 6))
plt.plot(fpr_train_rf, tpr_train_rf, label=f"AUC = {roc_auc_score(y_train, y_train_proba_rf):.2f}")
plt.plot([0, 1], [0, 1], linestyle='--', label='Random Guess')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC Curve - Tuned Random Forest (Train)')
plt.legend(loc='lower right')
plt.grid(True)
plt.show()

# Predict
y_pred_best_rf = best_rf.predict(X_test)
y_proba_best_rf = best_rf.predict_proba(X_test)[:, 1]

# Metrics
print("Classification Report (Tuned Random Forest - Test):")
print(classification_report(y_test, y_pred_best_rf))

print("Accuracy:", round(accuracy_score(y_test, y_pred_best_rf), 2))
print("Precision:", round(precision_score(y_test, y_pred_best_rf), 2))
print("Recall:", round(recall_score(y_test, y_pred_best_rf), 2))
print("F1 Score:", round(f1_score(y_test, y_pred_best_rf), 2))
print("AUC Score:", round(roc_auc_score(y_test, y_proba_best_rf), 2))

# Confusion Matrix
sns.heatmap(confusion_matrix(y_test, y_pred_best_rf), annot=True, fmt='d', cmap='Blues',
            xticklabels=['No Default', 'Default'], yticklabels=['No Default', 'Default'])
plt.title("Confusion Matrix - Tuned Random Forest (Test)")
plt.xlabel("Predicted Label")
plt.ylabel("True Label")
plt.show()

# ROC Curve
fpr, tpr, _ = roc_curve(y_test, y_proba_best_rf)
plt.figure(figsize=(8, 6))
plt.plot(fpr, tpr, label=f'AUC = {roc_auc_score(y_test, y_proba_best_rf):.2f}')
plt.plot([0, 1], [0, 1], linestyle='--', label='Random Guess')
plt.xlabel("False Positive Rate")
plt.ylabel("True Positive Rate")
plt.title("ROC Curve - Tuned Random Forest")
plt.legend()
plt.grid(True)
plt.show()
# Save metrics for comparison table later
rf_tuned_metrics = {
    'model': 'Tuned Random Forest',
    'accuracy': round(accuracy_score(y_test, y_pred_best_rf), 2),
    'precision': round(precision_score(y_test, y_pred_best_rf), 2),
    'recall': round(recall_score(y_test, y_pred_best_rf), 2),
    'f1': round(f1_score(y_test, y_pred_best_rf), 2),
    'auc': round(roc_auc_score(y_test, y_proba_best_rf), 2)
}

# feature importance
importances = best_rf.named_steps['classifier'].feature_importances_
feature_names = num_cols + cat_cols
importances_df = pd.DataFrame({'Feature': feature_names, 'Importance': importances})
importances_df = importances_df.sort_values(by='Importance', ascending=False)
plt.figure(figsize=(10, 6))
sns.barplot(x='Importance', y='Feature', data=importances_df)
plt.title("Feature Importance - Tuned Random Forest")
plt.xlabel("Importance")
plt.ylabel("Feature")
plt.show()

def plot_threshold_curves(metrics_df, model_name="Model"):
    plt.figure(figsize=(10,6))
    plt.plot(metrics_df['Threshold'], metrics_df['Precision'], label='Precision')
    plt.plot(metrics_df['Threshold'], metrics_df['Recall'], label='Recall')
    plt.plot(metrics_df['Threshold'], metrics_df['F1'], label='F1 Score')
    plt.plot(metrics_df['Threshold'], metrics_df['Accuracy'], label='Accuracy')
    plt.xlabel("Threshold")
    plt.ylabel("Score")
    plt.title(f"Threshold vs Metrics - {model_name}")
    plt.legend()
    plt.grid(True)
    plt.show()

# Plot for RF
plot_threshold_curves(threshold_metrics_rf, model_name="Tuned Random Forest")

# Plot for DT
plot_threshold_curves(threshold_metrics_dt, model_name="Tuned Decision Tree")

rf_probs = best_rf.predict_proba(X_test)[:, 1]  # Probabilities for class 1 (default)
best_thresh_rf, rf_thresh_df = optimal_threshold(rf_probs, y_test, model_name="Tuned Random Forest")

dt_probs = best_dt.predict_proba(X_test)[:, 1]  # Probabilities for class 1 (default)
best_thresh_dt, dt_thresh_df = optimal_threshold(dt_probs, y_test, model_name="Tuned Decision Tree")

# Combine saved metrics
all_metrics = pd.DataFrame([
    logreg_metrics,
    dt_metrics,
    dt_tuned_metrics,
    rf_metrics,
    rf_tuned_metrics
])

# Set model names as index
all_metrics.set_index('model', inplace=True)

# Plotting
plt.figure(figsize=(12, 6))
all_metrics.plot(kind='bar', figsize=(12, 6), colormap='Set2')

plt.title('Model Performance Comparison (Test Set)', fontsize=16)
plt.ylabel('Score')
plt.xticks(rotation=45)
plt.ylim(0, 1)
plt.legend(loc='lower right')
plt.grid(axis='y')
plt.tight_layout()
plt.show()